package com.chance.cc.crawler.development.bootstrap.dxy;

import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;

/**
 * @author lt
 * @version 1.0
 * @date 2021-02-04 18:29:50
 * @email okprog@sina.com
 */
public class DXYStart {
    private static final String domain = "dxy";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        //HL89Q19E86E2987D
        //71F33D94CE5F7BF2
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static void main(String[] args) {
        //设置https协议访问
        System.setProperty("https.protocols", "TLSv1,TLSv1.1,TLSv1.2,SSLv3");
        //关键词 source record
        CrawlerRequestRecord keywordCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain + "_keywords",turnPageItem)
                .httpUrl("http://192.168.1.215:9599//v1/meta/"+domain+"/keys?site=search")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        //文章采集
        CrawlerRequestRecord articleCrawler = doSearchNewsCrawler();
        //文章采集
        CrawlerRequestRecord forumCrawler = doSearchBBSCrawler();



        DevCrawlerController.builder()
                .triggerInfo(domain,domain + "_trigger",System.currentTimeMillis(),domain+ "_job")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain)) //内存队列
                .consoleResultPipeline() //控制台输t出
                .fileResultPipeline("D:\\chance\\data\\dxy\\dxy_article_test.json",false)
//                .requestRecord(articleCrawler)  //more job
                .requestRecord(forumCrawler)  //more job
                .supportRecord(keywordCrawlerRecord)
                .crawlerThreadNum(10)
                .build()
                .start();
    }

    public static CrawlerRequestRecord doSearchNewsCrawler(){
        String site = "news";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .recordKey("https://portal.dxy.cn/")
                .httpUrl("https://portal.dxy.cn/")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*7,null))
                .proxy(proxy)
                .needParsed(false)
                .needWashed(false)
                .build();

        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    public static CrawlerRequestRecord doSearchBBSCrawler(){
        String site = "forum";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .recordKey("https://portal.dxy.cn/")
                .httpUrl("https://portal.dxy.cn/")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*7,null))
                .proxy(proxy)
                .needParsed(false)
                .needWashed(false)
                .build();

        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }


}
